{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "solving pendulum using actor-critic model\n",
    "\"\"\"\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Input, GRU\n",
    "from tensorflow.keras.layers import Add, Concatenate\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import random\n",
    "from collections import deque\n",
    "\n",
    "def stack_samples(samples):\n",
    "\t\n",
    "\t\n",
    "\tcurrent_states = [train_data[0] for train_data in samples]\n",
    "\tactions = [train_data[1] for train_data in samples]\n",
    "\trewards = [train_data[2] for train_data in samples]\n",
    "\tnew_states = [train_data[3] for train_data in samples]\n",
    "\tdones = [train_data[4] for train_data in samples]\n",
    "\t\n",
    "\treturn current_states, actions, rewards, new_states, dones\n",
    "\t\n",
    "\n",
    "# determines how to assign values to each state, i.e. takes the state\n",
    "# and action (two-input model) and determines the corresponding value\n",
    "class ActorCritic:\n",
    "\tdef __init__(self, env, sess):\n",
    "\t\tself.env  = env\n",
    "\t\tself.sess = sess\n",
    "\n",
    "\t\tself.learning_rate = 0.0001\n",
    "\t\tself.epsilon = .9\n",
    "\t\tself.epsilon_decay = .99995\n",
    "\t\tself.gamma = .90\n",
    "\t\tself.tau   = .01\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                               Actor Model                             #\n",
    "\t\t# Chain rule: find the gradient of chaging the actor network params in  #\n",
    "\t\t# getting closest to the final value network predictions, i.e. de/dA    #\n",
    "\t\t# Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act #\n",
    "\t\t# ===================================================================== #\n",
    "\n",
    "\t\tself.memory = deque(maxlen=4000)\n",
    "\t\tself.actor_state_input, self.actor_model = self.create_actor_model()\n",
    "\t\t_, self.target_actor_model = self.create_actor_model()\n",
    "\n",
    "\t\tself.actor_critic_grad = tf.placeholder(tf.float32,\n",
    "\t\t\t[None, self.env.action_space.shape[0]]) # where we will feed de/dC (from critic)\n",
    "\n",
    "\t\tactor_model_weights = self.actor_model.trainable_weights\n",
    "\t\tself.actor_grads = tf.gradients(self.actor_model.output,\n",
    "\t\t\tactor_model_weights, -self.actor_critic_grad) # dC/dA (from actor)\n",
    "\t\tgrads = zip(self.actor_grads, actor_model_weights)\n",
    "\t\tself.optimize = tf.train.AdamOptimizer(self.learning_rate).apply_gradients(grads)\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                              Critic Model                             #\n",
    "\t\t# ===================================================================== #\n",
    "\n",
    "\t\tself.critic_state_input, self.critic_action_input, \\\n",
    "\t\t\tself.critic_model = self.create_critic_model()\n",
    "\t\t_, _, self.target_critic_model = self.create_critic_model()\n",
    "\n",
    "\t\tself.critic_grads = tf.gradients(self.critic_model.output,\n",
    "\t\t\tself.critic_action_input) # where we calcaulte de/dC for feeding above\n",
    "\n",
    "\t\t# Initialize for later gradient calculations\n",
    "\t\tself.sess.run(tf.initialize_all_variables())\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Definitions                            #\n",
    "\n",
    "\tdef create_actor_model(self):\n",
    "\t# ========================================================================= #\n",
    "\t\tstate_input = Input(shape=(None,self.env.observation_space.shape[0]))\n",
    "\t\th1 = Dense(500, activation='relu')(state_input)\n",
    "\t\tactor_rnn,state_h = GRU(256, return_state=True)(h1)\n",
    "\t\th2 = Dense(500, activation='relu')(state_h)\n",
    "\t\toutput = Dense(self.env.action_space.shape[0], activation='tanh')(h2)\n",
    "\n",
    "\t\tmodel = Model([state_input], output)\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, model\n",
    "\n",
    "\tdef create_critic_model(self):\n",
    "\t\tstate_input = Input(shape=(None,self.env.observation_space.shape[0]))\n",
    "\t\tstate_h1 = Dense(500, activation='relu')(state_input)\n",
    "\t\tcritic_rnn,state_h2 = GRU(256, return_state=True)(state_h1)\n",
    "\n",
    "\t\taction_input = Input(shape=self.env.action_space.shape)\n",
    "\t\taction_h1    = Dense(500)(action_input)\n",
    "\n",
    "\t\tmerged    = Concatenate()([state_h2, action_h1])\n",
    "\t\tmerged_h1 = Dense(500, activation='relu')(merged)\n",
    "\t\toutput = Dense(1, activation='linear')(merged_h1)\n",
    "\t\tmodel  = Model([state_input,action_input],output)\n",
    "\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, action_input, model\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                               Model Training                              #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef remember(self, cur_state, action, reward, new_state, done):\n",
    "\t\tself.memory.append([cur_state, action, reward, new_state, done])\n",
    "\n",
    "\tdef _train_actor(self, samples):\n",
    "\t\t\n",
    "\t\tcur_states, actions, rewards, new_seqs, _ =  stack_samples(samples)\n",
    "\t\tpredicted_actions = [self.actor_model.predict(cur_state) for cur_state in cur_states]\n",
    "\t\tfor i in range(len(cur_states)):\n",
    "\t\t    grads = self.sess.run(self.critic_grads, feed_dict={\n",
    "\t\t        self.critic_state_input:  cur_states[i],\n",
    "\t\t        self.critic_action_input: predicted_actions[i]\n",
    "\t\t    })[0]\n",
    "\n",
    "\t\t    self.sess.run(self.optimize, feed_dict={\n",
    "\t\t        self.actor_state_input: cur_states[i],\n",
    "\t\t        self.actor_critic_grad: grads\n",
    "\t\t    })\n",
    "\n",
    "\tdef _train_critic(self, samples):\n",
    "   \n",
    "\n",
    "\t\tcur_states, actions, rewards, new_states, dones = stack_samples(samples)\n",
    "\t\ttarget_actions = [self.target_actor_model.predict(new_state) for new_state in new_states]\n",
    "\t\tfuture_rewards = [self.target_critic_model.predict([new_states[i], target_actions[i]]) for i in range(len(new_states))]\n",
    "\t\trewards = np.asarray(rewards)\n",
    "\t\tfuture_rewards = np.asarray(future_rewards).reshape(rewards.shape)\n",
    "\t\t#print(future_rewards.shape)\n",
    "\t\tdones = np.asarray(dones).reshape(rewards.shape)\n",
    "\t\t#print(dones.shape)\n",
    "\t\trewards += self.gamma * future_rewards* (1 - dones)\n",
    "\t\t\n",
    "\t\tfor i in range(len(cur_states)):\n",
    "\t\t    evaluation = self.critic_model.fit([cur_states[i], actions[i]], rewards[i], verbose=0)\n",
    "\t\t#print(evaluation.history)\n",
    "\tdef train(self):\n",
    "\t\tbatch_size = 64\n",
    "\t\tif len(self.memory) < batch_size:\n",
    "\t\t\treturn\n",
    "\n",
    "\t\trewards = []\n",
    "\t\tsamples = random.sample(self.memory, batch_size)\n",
    "\t\tself.samples = samples\n",
    "\t\tself._train_critic(samples)\n",
    "\t\tself._train_actor(samples)\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                         Target Model Updating                             #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef _update_actor_target(self):\n",
    "\t\tactor_model_weights  = self.actor_model.get_weights()\n",
    "\t\tactor_target_weights = self.target_actor_model.get_weights()\n",
    "\t\t\n",
    "\t\tfor i in range(len(actor_target_weights)):\n",
    "\t\t\tactor_target_weights[i] = actor_model_weights[i]*self.tau + actor_target_weights[i]*(1-self.tau)\n",
    "\t\tself.target_actor_model.set_weights(actor_target_weights)\n",
    "\n",
    "\tdef _update_critic_target(self):\n",
    "\t\tcritic_model_weights  = self.critic_model.get_weights()\n",
    "\t\tcritic_target_weights = self.target_critic_model.get_weights()\n",
    "\t\t\n",
    "\t\tfor i in range(len(critic_target_weights)):\n",
    "\t\t\tcritic_target_weights[i] = critic_model_weights[i]*self.tau + critic_target_weights[i]*(1-self.tau)\n",
    "\t\tself.target_critic_model.set_weights(critic_target_weights)\n",
    "\n",
    "\tdef update_target(self):\n",
    "\t\tself._update_actor_target()\n",
    "\t\tself._update_critic_target()\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Predictions                            #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef act(self, cur_state):\n",
    "\t\tself.epsilon *= self.epsilon_decay\n",
    "\t\tif np.random.random() < self.epsilon:\n",
    "\t\t\treturn self.actor_model.predict(cur_state)*2 + np.random.normal()\n",
    "\t\treturn self.actor_model.predict(cur_state)*2\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_should_use.py:193: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "trial:0\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-10.583074]\n",
      "trial:1\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-9.348821]\n",
      "trial:2\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-8.211534]\n",
      "trial:3\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-19.40782]\n",
      "trial:4\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-1.4995368]\n",
      "trial:5\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-3.3849335]\n",
      "trial:6\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "[-26.510612]\n",
      "trial:7\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n",
      "(64, 1)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-7fd7f930efa2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"__main__\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m         \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-7fd7f930efa2>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-091e1345739d>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    155\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msamples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_critic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_actor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0;31m# ========================================================================= #\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-091e1345739d>\u001b[0m in \u001b[0;36m_train_actor\u001b[0;34m(self, samples)\u001b[0m\n\u001b[1;32m    127\u001b[0m \t\t    self.sess.run(self.optimize, feed_dict={\n\u001b[1;32m    128\u001b[0m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor_state_input\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcur_states\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor_critic_grad\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mgrads\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m \t\t    })\n\u001b[1;32m    131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1427\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1431\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def main():\n",
    "\tsess = tf.Session()\n",
    "\tK.set_session(sess)\n",
    "\tenv = gym.make(\"Pendulum-v0\")\n",
    "\tactor_critic = ActorCritic(env, sess)\n",
    "\n",
    "\tnum_trials = 10000\n",
    "\ttrial_len  = 200\n",
    "\n",
    "\tfor i in range(num_trials):\n",
    "\t\tprint(\"trial:\" + str(i))\n",
    "\t\tcur_state = env.reset()\n",
    "\t\taction = env.action_space.sample()\n",
    "\t\treward_sum = 0\n",
    "\t\tobs_list = []        \n",
    "\t\tobs_list.append(cur_state)\n",
    "\t\tfor j in range(trial_len):\n",
    "\t\t\t#env.render()        \n",
    "\t\t\tobs_seq = np.asarray(obs_list)\n",
    "\t\t\tobs_seq = obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "\t\t\taction = actor_critic.act(obs_seq)\n",
    "\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\treward += reward\n",
    "\t\t\tif j == (trial_len - 1):\n",
    "\t\t\t\tdone = True\n",
    "\t\t\t\tprint(reward)\n",
    "\n",
    "\t\t\tif (j % 5 == 0):\n",
    "\t\t\t\tactor_critic.train()\n",
    "\t\t\t\tactor_critic.update_target()   \n",
    "\t\t\t\n",
    "\t\t\tnew_state = new_state.reshape((env.observation_space.shape))\n",
    "\n",
    "\t\t\tobs_list.append(new_state)\n",
    "\t\t\tnext_obs_seq = np.asarray(obs_list)\n",
    "\t\t\tnext_obs_seq = next_obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "\t\t\tactor_critic.remember(obs_seq, action, reward, next_obs_seq, done)\n",
    "\t\t\tcur_state = new_state\n",
    "\n",
    "\t\tif (i % 5 == 0) and i!=0:\n",
    "\t\t\tcur_state = env.reset()\n",
    "\t\t\tobs_list = []                   \n",
    "\t\t\tfor j in range(500):\n",
    "\t\t\t\t#env.render()\n",
    "\t\t\t\tcur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "\t\t\t\tobs_list.append(cur_state)\n",
    "\t\t\t\tobs_seq = np.asarray(obs_list)\n",
    "\t\t\t\tobs_seq = obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "\t\t\t\taction = actor_critic.act(obs_seq)\n",
    "\t\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\t\t#reward += reward\n",
    "\t\t\t\t#if j == (trial_len - 1):\n",
    "\t\t\t\t\t#done = True\n",
    "\t\t\t\t\t#print(reward)\n",
    "\n",
    "\t\t\t\t#if (j % 5 == 0):\n",
    "\t\t\t\t#    actor_critic.train()\n",
    "\t\t\t\t#    actor_critic.update_target()   \n",
    "\t\t\t\t\n",
    "\t\t\t\tnew_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "\t\t\t\t#actor_critic.remember(cur_state, action, reward, new_state, done)\n",
    "\t\t\t\tcur_state = new_state\n",
    "\t\t\t\t\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
